import numpy as np
import pandas as pd
import time

np.random.seed(2)  # 伪随机数
N_STATES = 6  #
ACTIONS = ['left', 'right']
EPSILON = 0.9  # 最优
ALPHA = 0.1  # 学习率
LAMBDA = 0.9  # 衰减值
MAX_EPSILONS = 7  # 13次
FRESH_TIME = 0.0  # 每一步时间


def build_q_table(n_states, actions):
    table = pd.DataFrame(np.zeros((n_states, len(actions))), columns=actions)
    return table


def choose_action(state, q_table):
    """选动作"""
    state_actions = q_table.iloc[state, :]
    if np.random.uniform() > EPSILON or state_actions.all() == 0:
        action_name = np.random.choice(ACTIONS)
    else:
        action_name = state_actions.idxmax()
    return action_name


def get_env_feedback(s, a):
    if a == 'right':
        if s == N_STATES - 2:
            s_ = 'terminal'
            r = 1
        else:
            s_ = s + 1
            r = 0
    else:
        r = 0
        if s == 0:
            s_ = s
        else:
            s_ = s - 1
    return s_, r


def update_env(s, episode, step_counter):
    env_list = ['-'] * (N_STATES - 1) + ['T']
    if s == 'terminal':
        interaction = f'\tepisode={episode + 1},step={step_counter}'
        print(interaction, end='')
        time.sleep(2)
    else:
        env_list[s] = 'o'
        interaction = ''.join(env_list)
        print("\r" + interaction, end='')
        time.sleep(FRESH_TIME)


def r_learn():
    q_table = build_q_table(N_STATES, ACTIONS)
    for episode in range(MAX_EPSILONS):
        step_count, s = 0, 0
        is_terminated = False
        update_env(s, episode, step_count)
        while not is_terminated:
            a = choose_action(s, q_table)
            s_, r = get_env_feedback(s, a)
            q_predict = q_table.loc[s, a]
            if s_ != 'terminal':
                q_target = r + LAMBDA * q_table.iloc[s_, :].max()
            else:
                q_target = r
                is_terminated = True
            q_table.loc[s, a] += ALPHA * (q_target - q_predict)
            s = s_
            update_env(s, episode, step_count + 1)
            step_count += 1
    return q_table


if __name__ == '__main__':
    q = r_learn()
    print(q)
